package com.itcast.clientx.processor;

import us.codecraft.webmagic.Page;
import us.codecraft.webmagic.Site;
import us.codecraft.webmagic.Spider;
import us.codecraft.webmagic.pipeline.FilePipeline;
import us.codecraft.webmagic.processor.PageProcessor;

public class PageProcess implements PageProcessor {

    private Site site = Site.me();
    @Override
    public void process(Page page) {
        page.putField("div",page.getHtml().css("li").all());
        /*page.addTargetRequests(page.getHtml().css("ul.cate_menu li a").links().regex(".*html$").all());
        page.putField("detail",page.getHtml().css("a").all());*/

    }

    @Override
    public Site getSite() {
        site.setCharset("utf8").setTimeOut(10*1000).setRetrySleepTime(3000).setRetryTimes(3);
        site.addHeader("Referrer Policy","no-referrer-when-downgrade").addHeader("Remote Address","104.26.0.44:443");

        site.setDomain("mvnrepository.com");
        site.setUserAgent("Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/70.0.3538.25 Safari/537.36 Core/1.70.3868.400 QQBrowser/10.8.4394.400");
        site.addCookie("cookie","_ga=GA1.2.104143695.1620453605; __gads=ID=b38aaa7b11b51a40-2280bbe9e0c700fc:T=1620455551:RT=1620455551:S=ALNI_MaCwLXxXyg0DwHiTyJ692bfuXLMDQ; __cfduid=d6824abadd7573d53b093425292f04aa41620546458; cf_chl_prog=a11; cf_clearance=56c151ce5967974d5c00d2dd8da693b8491e99f1-1620952123-0-250; _gid=GA1.2.1739845534.1620952101; _gat=1; __cf_bm=2e081d4f7101b499db05280dd7db99a337322723-1620952124-1800-AUb/uEos+ncW/rwIr1qcnXY4JSHo2RBkpKxJQzP11AaceS6UIbblZC77lESiJGaiC4ZMitORg8eBfmC+L/dFwP60WpdyJmRoNPm+rivpwP415gRaPaBMt0xgKJhnx8s9nw==; MVN_SESSION=eyJhbGciOiJIUzI1NiJ9.eyJkYXRhIjp7InVpZCI6IjY1YTU3MDYxLWI0NGItMTFlYi1hNWU5LTMxODk0NzRjNzE2ZCJ9LCJleHAiOjE2NTI0ODgxNDgsIm5iZiI6MTYyMDk1MjE0OCwiaWF0IjoxNjIwOTUyMTQ4fQ._XY2Psx2EtsrCN9MdrlkJIunoDGW9id7ygl2_FCwaok");
        return site;
    }

    public static void main(String[] args) {
        Spider.create(new PageProcess()).addUrl("https://mvnrepository.com/search?q=poi").run();
    }
}
